{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "ba51f09f-fec0-4c71-a2e7-3900805daad8",
   "metadata": {},
   "source": [
    "----------\n",
    "* 时间：2023/10/11\n",
    "* WEB数据挖掘\n",
    "* week06\n",
    "--------------"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c35bbf85-f271-4cb9-8f0c-7de93e717db1",
   "metadata": {},
   "source": [
    "# 1.本周内容（动态加载Ajax）数据获取\n",
    "## 1.1 实践项目：liepin数据抓取\n",
    "> 1.网站：liepin.com"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "8b4d7019-fae8-45c3-893a-f34a735d8b5f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 尝试 request_html模块\n",
    "from requests_html import HTMLSession"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "1e878597-65e3-4bcc-973e-1386e67772b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "session = HTMLSession()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "3d13927a-58ac-4f8b-a072-453fd6e0b928",
   "metadata": {},
   "outputs": [],
   "source": [
    "r = session.get( 'https://www.liepin.com/zhaopin/?currentPage=0&pageSize=40&city=050090&dq=050090&pubTime=&key=%E4%BA%A7%E5%93%81%E7%BB%8F%E7%90%86&suggestTag=&otherCity=&industry=&ckId=62e7o1z9y36cl5bo4i2qdk38zjk0n7cb&scene=condition&skId=62e7o1z9y36cl5bo4i2qdk38zjk0n7cb&fkId=62e7o1z9y36cl5bo4i2qdk38zjk0n7cb&sfrom=search_job_pc&suggestId=')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "ca446326-1182-4f0a-a54f-b2217a1d0989",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<Element 'text' font-family='PingFangSC-Medium, PingFang SC' font-size='10' font-weight='400' fill='#FFF'>]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r.html.find(\"text\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "95c2a2be-8244-492c-b45d-302940143443",
   "metadata": {},
   "outputs": [],
   "source": [
    "r = session.post('https://api-c.liepin.com/api/com.liepin.searchfront4c.pc-search-job')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "fe55f184-1237-439d-8382-519e26446ba5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "200"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r.status_code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "02ccf0dd-4d5c-4035-b53c-5c99a0ad05bb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'flag': 0, 'code': '-1400', 'msg': '出错了（400）！'}"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r.json()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "e53d59b3-f060-4be1-a96b-653628fe6623",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/603.3.8 (KHTML, like Gecko) Version/10.1.2 Safari/603.3.8', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive'}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r.request.headers"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e9b71eaa-9e5c-4c80-948e-91bf57bec610",
   "metadata": {},
   "source": [
    "* 无法请求到数据的原因：\n",
    "> 1.模拟用户真实代理           \n",
    "> 2.提供请求的参数（类比API）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "aa869dcc-6355-4b08-884c-ba3202bb0a6c",
   "metadata": {},
   "outputs": [],
   "source": [
    "liepin_headers ={\n",
    "  \"Accept:\",\n",
    "  \"application/json, text/plain, */*\",\n",
    "  \"Accept-Encoding:\",\n",
    "  \"gzip, deflate, br\",\n",
    "  \"Accept-Language:\",\n",
    "  \"zh-CN,zh;q=0.9\",\n",
    "  \"Connection:\",\n",
    "  \"keep-alive\",\n",
    "  \"Content-Length:\",\n",
    "  \"488\",\n",
    "  \"Content-Type:\",\n",
    "  \"application/json;charset=UTF-8;\",\n",
    "  \"Cookie:\",\n",
    "  \"__uuid=1681902669177.40; __gc_id=db880718bc324e5ab4cd9eab373f6dff; _ga=GA1.1.986716534.1681902703; need_bind_tel=false; c_flag=0c1fa2488bed40c17275a1e48449b212; XSRF-TOKEN=fjf8WQc0S5GVngmQy3FweA; __tlog=1697021977511.09%7C00000000%7C00000000%7C00000000%7C00000000; Hm_lvt_a2647413544f5a04f00da7eee0d5e200=1697021978; __session_seq=6; __uv_seq=6; Hm_lpvt_a2647413544f5a04f00da7eee0d5e200=1697023064; __tlg_event_seq=91; acw_tc=2760829816970238790315481eda5eeca39119586d57c97bb1668daa6cce15; _ga_54YTJKWN86=GS1.1.1697021976.4.1.1697023907.0.0.0\",\n",
    "  \"Host:\",\n",
    "  \"api-c.liepin.com\",\n",
    "  \"Origin:\",\n",
    "  \"https://www.liepin.com\",\n",
    "  \"Referer:\",\n",
    "  \"https://www.liepin.com/\",\n",
    "  \"Sec-Ch-Ua:\",\n",
    "  \"\\\"Google Chrome\\\";v=\\\"117\\\", \\\"Not;A=Brand\\\";v=\\\"8\\\", \\\"Chromium\\\";v=\\\"117\\\"\",\n",
    "  \"Sec-Ch-Ua-Mobile:\",\n",
    "  \"?0\",\n",
    "  \"Sec-Ch-Ua-Platform:\",\n",
    "  \"\\\"Windows\\\"\",\n",
    "  \"Sec-Fetch-Dest:\",\n",
    "  \"empty\",\n",
    "  \"Sec-Fetch-Mode:\",\n",
    "  \"cors\",\n",
    "  \"Sec-Fetch-Site:\",\n",
    "  \"same-site\",\n",
    "  \"User-Agent:\",\n",
    "  \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36\",\n",
    "  \"X-Client-Type:\",\n",
    "  \"web\",\n",
    "  \"X-Fscp-Bi-Stat:\",\n",
    "  \"{\\\"location\\\": \\\"https://www.liepin.com/zhaopin/?city=050090&dq=050090&pubTime=&currentPage=0&pageSize=40&key=%E4%BA%A7%E5%93%81%E7%BB%8F%E7%90%86&suggestTag=&workYearCode=0&compId=&compName=&compTag=&industry=&salary=&jobKind=&compScale=&compKind=&compStage=&eduLevel=&otherCity=&ckId=bhmi0ix1szqb0yj4533vr3yys8x301da&scene=condition&skId=bhmi0ix1szqb0yj4533vr3yys8x301da&fkId=bhmi0ix1szqb0yj4533vr3yys8x301da&sfrom=search_job_pc&suggestId=\\\"}\",\n",
    "  \"X-Fscp-Fe-Version:\",\n",
    "  \"X-Fscp-Std-Info:\",\n",
    "  \"{\\\"client_id\\\": \\\"40108\\\"}\",\n",
    "  \"X-Fscp-Trace-Id:\",\n",
    "  \"35a4b98c-b082-43b9-9c0c-64fa5f761f30\",\n",
    "  \"X-Fscp-Version:\",\n",
    "  \"1.1\",\n",
    "  \"X-Requested-With:\",\n",
    "  \"XMLHttpRequest\",\n",
    "  \"X-Xsrf-Token:\",\n",
    "  \"fjf8WQc0S5GVngmQy3FweA\"\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "af585bcf-4d70-40ec-823b-b152684079c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "payload = {\n",
    "\t\"data\": {\n",
    "\t\t\"mainSearchPcConditionForm\": {\n",
    "\t\t\t\"city\": \"050090\",\n",
    "\t\t\t\"dq\": \"050090\",\n",
    "\t\t\t\"pubTime\": \"\",\n",
    "\t\t\t\"currentPage\": \"0\",\n",
    "\t\t\t\"pageSize\": 40,\n",
    "\t\t\t\"key\": \"产品经理\",\n",
    "\t\t\t\"suggestTag\": \"\",\n",
    "\t\t\t\"workYearCode\": \"0\",\n",
    "\t\t\t\"compId\": \"\",\n",
    "\t\t\t\"compName\": \"\",\n",
    "\t\t\t\"compTag\": \"\",\n",
    "\t\t\t\"industry\": \"\",\n",
    "\t\t\t\"salary\": \"\",\n",
    "\t\t\t\"jobKind\": \"\",\n",
    "\t\t\t\"compScale\": \"\",\n",
    "\t\t\t\"compKind\": \"\",\n",
    "\t\t\t\"compStage\": \"\",\n",
    "\t\t\t\"eduLevel\": \"\"\n",
    "\t\t},\n",
    "\t\t\"passThroughForm\": {\n",
    "\t\t\t\"scene\": \"condition\",\n",
    "\t\t\t\"skId\": \"bhmi0ix1szqb0yj4533vr3yys8x301da\",\n",
    "\t\t\t\"fkId\": \"bhmi0ix1szqb0yj4533vr3yys8x301da\",\n",
    "\t\t\t\"ckId\": \"k6p23gvq6wgpym84pqi3dzgnjwf77kh9\",\n",
    "\t\t\t\"suggest\": None\n",
    "\t\t}\n",
    "\t}\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "ab2fa422-59f1-4204-80ec-c761d2e12619",
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'set' object has no attribute 'items'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[41], line 2\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mjson\u001b[39;00m\n\u001b[1;32m----> 2\u001b[0m r \u001b[38;5;241m=\u001b[39m \u001b[43msession\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpost\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m      3\u001b[0m \u001b[43m            \u001b[49m\u001b[43murl\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mhttps://api-c.liepin.com/api/com.liepin.searchfront4c.pc-search-job\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[0;32m      4\u001b[0m \u001b[43m            \u001b[49m\u001b[43mdata\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mjson\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdumps\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpayload\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;66;43;03m#POST请求带的参数\u001b[39;49;00m\n\u001b[0;32m      5\u001b[0m \u001b[43m            \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mliepin_headers\u001b[49m\u001b[43m   \u001b[49m\u001b[38;5;66;43;03m# 模拟真实用户\u001b[39;49;00m\n\u001b[0;32m      6\u001b[0m \u001b[43m)\u001b[49m\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\requests\\sessions.py:637\u001b[0m, in \u001b[0;36mSession.post\u001b[1;34m(self, url, data, json, **kwargs)\u001b[0m\n\u001b[0;32m    626\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mpost\u001b[39m(\u001b[38;5;28mself\u001b[39m, url, data\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, json\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[0;32m    627\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124mr\u001b[39m\u001b[38;5;124;03m\"\"\"Sends a POST request. Returns :class:`Response` object.\u001b[39;00m\n\u001b[0;32m    628\u001b[0m \n\u001b[0;32m    629\u001b[0m \u001b[38;5;124;03m    :param url: URL for the new :class:`Request` object.\u001b[39;00m\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    634\u001b[0m \u001b[38;5;124;03m    :rtype: requests.Response\u001b[39;00m\n\u001b[0;32m    635\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[1;32m--> 637\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mrequest(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mPOST\u001b[39m\u001b[38;5;124m\"\u001b[39m, url, data\u001b[38;5;241m=\u001b[39mdata, json\u001b[38;5;241m=\u001b[39mjson, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\requests\\sessions.py:575\u001b[0m, in \u001b[0;36mSession.request\u001b[1;34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001b[0m\n\u001b[0;32m    562\u001b[0m \u001b[38;5;66;03m# Create the Request.\u001b[39;00m\n\u001b[0;32m    563\u001b[0m req \u001b[38;5;241m=\u001b[39m Request(\n\u001b[0;32m    564\u001b[0m     method\u001b[38;5;241m=\u001b[39mmethod\u001b[38;5;241m.\u001b[39mupper(),\n\u001b[0;32m    565\u001b[0m     url\u001b[38;5;241m=\u001b[39murl,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    573\u001b[0m     hooks\u001b[38;5;241m=\u001b[39mhooks,\n\u001b[0;32m    574\u001b[0m )\n\u001b[1;32m--> 575\u001b[0m prep \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mprepare_request\u001b[49m\u001b[43m(\u001b[49m\u001b[43mreq\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    577\u001b[0m proxies \u001b[38;5;241m=\u001b[39m proxies \u001b[38;5;129;01mor\u001b[39;00m {}\n\u001b[0;32m    579\u001b[0m settings \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmerge_environment_settings(\n\u001b[0;32m    580\u001b[0m     prep\u001b[38;5;241m.\u001b[39murl, proxies, stream, verify, cert\n\u001b[0;32m    581\u001b[0m )\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\requests\\sessions.py:486\u001b[0m, in \u001b[0;36mSession.prepare_request\u001b[1;34m(self, request)\u001b[0m\n\u001b[0;32m    483\u001b[0m     auth \u001b[38;5;241m=\u001b[39m get_netrc_auth(request\u001b[38;5;241m.\u001b[39murl)\n\u001b[0;32m    485\u001b[0m p \u001b[38;5;241m=\u001b[39m PreparedRequest()\n\u001b[1;32m--> 486\u001b[0m \u001b[43mp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mprepare\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m    487\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmethod\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmethod\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mupper\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    488\u001b[0m \u001b[43m    \u001b[49m\u001b[43murl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    489\u001b[0m \u001b[43m    \u001b[49m\u001b[43mfiles\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfiles\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    490\u001b[0m \u001b[43m    \u001b[49m\u001b[43mdata\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    491\u001b[0m \u001b[43m    \u001b[49m\u001b[43mjson\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mjson\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    492\u001b[0m \u001b[43m    \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmerge_setting\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m    493\u001b[0m \u001b[43m        \u001b[49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdict_class\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mCaseInsensitiveDict\u001b[49m\n\u001b[0;32m    494\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    495\u001b[0m \u001b[43m    \u001b[49m\u001b[43mparams\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmerge_setting\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mparams\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mparams\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    496\u001b[0m \u001b[43m    \u001b[49m\u001b[43mauth\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmerge_setting\u001b[49m\u001b[43m(\u001b[49m\u001b[43mauth\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mauth\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    497\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcookies\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmerged_cookies\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    498\u001b[0m \u001b[43m    \u001b[49m\u001b[43mhooks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmerge_hooks\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mhooks\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mhooks\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    499\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    500\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m p\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\requests\\models.py:369\u001b[0m, in \u001b[0;36mPreparedRequest.prepare\u001b[1;34m(self, method, url, headers, files, data, params, auth, cookies, hooks, json)\u001b[0m\n\u001b[0;32m    367\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprepare_method(method)\n\u001b[0;32m    368\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprepare_url(url, params)\n\u001b[1;32m--> 369\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mprepare_headers\u001b[49m\u001b[43m(\u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    370\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprepare_cookies(cookies)\n\u001b[0;32m    371\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprepare_body(data, files, json)\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\requests\\models.py:489\u001b[0m, in \u001b[0;36mPreparedRequest.prepare_headers\u001b[1;34m(self, headers)\u001b[0m\n\u001b[0;32m    487\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mheaders \u001b[38;5;241m=\u001b[39m CaseInsensitiveDict()\n\u001b[0;32m    488\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m headers:\n\u001b[1;32m--> 489\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m header \u001b[38;5;129;01min\u001b[39;00m \u001b[43mheaders\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mitems\u001b[49m():\n\u001b[0;32m    490\u001b[0m         \u001b[38;5;66;03m# Raise exception on invalid header value.\u001b[39;00m\n\u001b[0;32m    491\u001b[0m         check_header_validity(header)\n\u001b[0;32m    492\u001b[0m         name, value \u001b[38;5;241m=\u001b[39m header\n",
      "\u001b[1;31mAttributeError\u001b[0m: 'set' object has no attribute 'items'"
     ]
    }
   ],
   "source": [
    "import json\n",
    "r = session.post(\n",
    "            url = 'https://api-c.liepin.com/api/com.liepin.searchfront4c.pc-search-job',\n",
    "            data = json.dumps(payload), #POST请求带的参数\n",
    "            headers = liepin_headers   # 模拟真实用户\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "5e5bf005-ca0d-4bf3-9fd4-68952e5531e3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<bound method Response.json of <Response [200]>>"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r.json"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "56bb6031-a770-473f-8a10-29e594fce684",
   "metadata": {},
   "source": [
    "* 笔记：\n",
    "* 动态页面获取关键步骤：\n",
    "> 1.如果获取不到数据，请首先做模拟用户，即模拟headers           \n",
    "> 2.检查F12Network部分XHR中参数的信息，确保与浏览器参数一致，先保证可以获取到数据         \n",
    "> 3.在2的基础上，根据我们的数据需求优化参数数据"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "316179ce-7d23-4eaf-9eae-0bcd025ad0c3",
   "metadata": {},
   "source": [
    "# 关于liepin核心参数的详细信息获取（帮助我们后续的分类查询）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "cd8fc47a-7746-4d79-b240-1fed3b37cddf",
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'set' object has no attribute 'items'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[43], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m r \u001b[38;5;241m=\u001b[39m \u001b[43msession\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpost\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m      2\u001b[0m \u001b[43m            \u001b[49m\u001b[43murl\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mhttps://api-c.liepin.com/api/com.liepin.searchfront4c.pc-search-job\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[0;32m      3\u001b[0m \u001b[43m            \u001b[49m\u001b[43mdata\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mjson\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdumps\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpayload\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m      4\u001b[0m \u001b[43m            \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mliepin_headers\u001b[49m\n\u001b[0;32m      5\u001b[0m \u001b[43m)\u001b[49m\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\requests\\sessions.py:637\u001b[0m, in \u001b[0;36mSession.post\u001b[1;34m(self, url, data, json, **kwargs)\u001b[0m\n\u001b[0;32m    626\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mpost\u001b[39m(\u001b[38;5;28mself\u001b[39m, url, data\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, json\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[0;32m    627\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124mr\u001b[39m\u001b[38;5;124;03m\"\"\"Sends a POST request. Returns :class:`Response` object.\u001b[39;00m\n\u001b[0;32m    628\u001b[0m \n\u001b[0;32m    629\u001b[0m \u001b[38;5;124;03m    :param url: URL for the new :class:`Request` object.\u001b[39;00m\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    634\u001b[0m \u001b[38;5;124;03m    :rtype: requests.Response\u001b[39;00m\n\u001b[0;32m    635\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[1;32m--> 637\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mrequest(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mPOST\u001b[39m\u001b[38;5;124m\"\u001b[39m, url, data\u001b[38;5;241m=\u001b[39mdata, json\u001b[38;5;241m=\u001b[39mjson, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\requests\\sessions.py:575\u001b[0m, in \u001b[0;36mSession.request\u001b[1;34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001b[0m\n\u001b[0;32m    562\u001b[0m \u001b[38;5;66;03m# Create the Request.\u001b[39;00m\n\u001b[0;32m    563\u001b[0m req \u001b[38;5;241m=\u001b[39m Request(\n\u001b[0;32m    564\u001b[0m     method\u001b[38;5;241m=\u001b[39mmethod\u001b[38;5;241m.\u001b[39mupper(),\n\u001b[0;32m    565\u001b[0m     url\u001b[38;5;241m=\u001b[39murl,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    573\u001b[0m     hooks\u001b[38;5;241m=\u001b[39mhooks,\n\u001b[0;32m    574\u001b[0m )\n\u001b[1;32m--> 575\u001b[0m prep \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mprepare_request\u001b[49m\u001b[43m(\u001b[49m\u001b[43mreq\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    577\u001b[0m proxies \u001b[38;5;241m=\u001b[39m proxies \u001b[38;5;129;01mor\u001b[39;00m {}\n\u001b[0;32m    579\u001b[0m settings \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmerge_environment_settings(\n\u001b[0;32m    580\u001b[0m     prep\u001b[38;5;241m.\u001b[39murl, proxies, stream, verify, cert\n\u001b[0;32m    581\u001b[0m )\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\requests\\sessions.py:486\u001b[0m, in \u001b[0;36mSession.prepare_request\u001b[1;34m(self, request)\u001b[0m\n\u001b[0;32m    483\u001b[0m     auth \u001b[38;5;241m=\u001b[39m get_netrc_auth(request\u001b[38;5;241m.\u001b[39murl)\n\u001b[0;32m    485\u001b[0m p \u001b[38;5;241m=\u001b[39m PreparedRequest()\n\u001b[1;32m--> 486\u001b[0m \u001b[43mp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mprepare\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m    487\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmethod\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmethod\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mupper\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    488\u001b[0m \u001b[43m    \u001b[49m\u001b[43murl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    489\u001b[0m \u001b[43m    \u001b[49m\u001b[43mfiles\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfiles\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    490\u001b[0m \u001b[43m    \u001b[49m\u001b[43mdata\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    491\u001b[0m \u001b[43m    \u001b[49m\u001b[43mjson\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mjson\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    492\u001b[0m \u001b[43m    \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmerge_setting\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m    493\u001b[0m \u001b[43m        \u001b[49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdict_class\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mCaseInsensitiveDict\u001b[49m\n\u001b[0;32m    494\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    495\u001b[0m \u001b[43m    \u001b[49m\u001b[43mparams\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmerge_setting\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mparams\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mparams\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    496\u001b[0m \u001b[43m    \u001b[49m\u001b[43mauth\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmerge_setting\u001b[49m\u001b[43m(\u001b[49m\u001b[43mauth\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mauth\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    497\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcookies\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmerged_cookies\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    498\u001b[0m \u001b[43m    \u001b[49m\u001b[43mhooks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmerge_hooks\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mhooks\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mhooks\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    499\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    500\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m p\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\requests\\models.py:369\u001b[0m, in \u001b[0;36mPreparedRequest.prepare\u001b[1;34m(self, method, url, headers, files, data, params, auth, cookies, hooks, json)\u001b[0m\n\u001b[0;32m    367\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprepare_method(method)\n\u001b[0;32m    368\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprepare_url(url, params)\n\u001b[1;32m--> 369\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mprepare_headers\u001b[49m\u001b[43m(\u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    370\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprepare_cookies(cookies)\n\u001b[0;32m    371\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprepare_body(data, files, json)\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\requests\\models.py:489\u001b[0m, in \u001b[0;36mPreparedRequest.prepare_headers\u001b[1;34m(self, headers)\u001b[0m\n\u001b[0;32m    487\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mheaders \u001b[38;5;241m=\u001b[39m CaseInsensitiveDict()\n\u001b[0;32m    488\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m headers:\n\u001b[1;32m--> 489\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m header \u001b[38;5;129;01min\u001b[39;00m \u001b[43mheaders\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mitems\u001b[49m():\n\u001b[0;32m    490\u001b[0m         \u001b[38;5;66;03m# Raise exception on invalid header value.\u001b[39;00m\n\u001b[0;32m    491\u001b[0m         check_header_validity(header)\n\u001b[0;32m    492\u001b[0m         name, value \u001b[38;5;241m=\u001b[39m header\n",
      "\u001b[1;31mAttributeError\u001b[0m: 'set' object has no attribute 'items'"
     ]
    }
   ],
   "source": [
    "r = session.post(\n",
    "            url = 'https://api-c.liepin.com/api/com.liepin.searchfront4c.pc-search-job',\n",
    "            data = json.dumps(payload),\n",
    "            headers = liepin_headers\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "2ecbfdfd-3be6-43b7-80fe-ca9ca714d308",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<bound method Response.json of <Response [200]>>"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "84088f86-d72c-4adf-8b2b-350383980d8d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'{\"flag\":0,\"code\":\"-1400\",\"msg\":\"出错了（400）！\"}'"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "95a92ac3-a6ed-4cfa-b6bd-4f18330ff6e9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 根据数据需求优化请求参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "e40d87d6-6ee0-46e1-a5ca-6d3684ce912c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a19e6bd0-f2aa-4a6e-8e28-243877040d3a",
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.json_normalize(liepin_data['data']['data'])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
